{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/ada/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input shape: (?, 141, 512)\n",
      "  [TL] InputLayer  lstm1_input: (?, 141, 256)\n",
      "  [TL] DynamicRNNLayer lstm_1: n_hidden:128, in_dim:3 in_shape:(?, 141, 256) cell_fn:BasicLSTMCell dropout:None n_layer:1\n",
      "       non specified batch_size, uses a tensor instead.\n",
      "  [TL] InputLayer  lstm2_input: (?, 141, 256)\n",
      "  [TL] DynamicRNNLayer lstm_2: n_hidden:128, in_dim:3 in_shape:(?, 141, 256) cell_fn:BasicLSTMCell dropout:None n_layer:1\n",
      "       non specified batch_size, uses a tensor instead.\n",
      "f_weights shape: (?, 141, 5)\n",
      "outputs shape: (?, 5)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorlayer as tl\n",
    "from tensorflow.contrib.keras import layers\n",
    "from tensorflow.contrib import slim\n",
    "import numpy as np\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "\n",
    "def weights_model(inputs, timesteps=101, dim=512, unit=128, emotion_embedding_dim=64, n_class=5):\n",
    "    x = tf.unstack(inputs, axis=1) # len(x)=timestaps\n",
    "    batch_norm_params = {\n",
    "        'decay': 0.99,\n",
    "        'epsilon': 0.001,\n",
    "        'updates_collections': None,\n",
    "        'variables_collections': [ tf.GraphKeys.TRAINABLE_VARIABLES ],\n",
    "    }\n",
    "    xs = []\n",
    "    for x_i in x:\n",
    "        xs.append(tf.contrib.layers.fully_connected(x_i, int(dim/2), activation_fn=tf.nn.relu,\\\n",
    "                                            normalizer_fn=slim.batch_norm, \\\n",
    "                                            normalizer_params=batch_norm_params, \\\n",
    "                                            scope=\"fc_1\", reuse=tf.AUTO_REUSE))\n",
    "    x = tf.stack(xs,axis=1)\n",
    "    x = tl.layers.InputLayer(x, name='lstm1_input')\n",
    "    x = tl.layers.DynamicRNNLayer(layer=x, cell_fn = tf.contrib.rnn.BasicLSTMCell, n_hidden = unit,\\\n",
    "                                  return_last = False, return_seq_2d = False, name = 'lstm_1').outputs\n",
    "    x = tf.reshape(x, (-1,timesteps,unit))\n",
    "\n",
    "    # FC (W^h) for mapping the dim from unit to emotion_embedding_dim\n",
    "    x = tf.unstack(x, axis=1)\n",
    "    xs = []\n",
    "    for x_i in x:\n",
    "        xs.append(tf.contrib.layers.fully_connected(x_i, emotion_embedding_dim, activation_fn=tf.nn.relu,\\\n",
    "                                            normalizer_fn=slim.batch_norm, \\\n",
    "                                            normalizer_params=batch_norm_params, \\\n",
    "                                            scope=\"fc_W\", reuse=tf.AUTO_REUSE))\n",
    "    \n",
    "    x = xs\n",
    "    # FC (e={e_1,e_2,...}) for embedding mapping from emotion_embedding_dim to n_class\n",
    "    xs = []\n",
    "    for x_i in x:\n",
    "        xs.append(tf.contrib.layers.fully_connected(x_i, n_class, activation_fn=tf.nn.softmax,\\\n",
    "                                            scope=\"fc_E\", reuse=tf.AUTO_REUSE))\n",
    "    x = tf.stack(xs,axis=1)\n",
    "\n",
    "    return x\n",
    "\n",
    "def attention_model(inputs, f, timesteps=101, dim=512, unit=128, n_class=5):\n",
    "    x = tf.unstack(inputs, axis=1) # len(x)=timestaps\n",
    "    batch_norm_params = {\n",
    "        'decay': 0.99,\n",
    "        'epsilon': 0.001,\n",
    "        'updates_collections': None,\n",
    "        'variables_collections': [ tf.GraphKeys.TRAINABLE_VARIABLES ],\n",
    "    }\n",
    "    xs = []\n",
    "    for x_i in x:\n",
    "        xs.append(tf.contrib.layers.fully_connected(x_i, int(dim/2), activation_fn=tf.nn.relu,\\\n",
    "                                            normalizer_fn=slim.batch_norm, \\\n",
    "                                            normalizer_params=batch_norm_params, \\\n",
    "                                            scope=\"fc_2\", reuse=tf.AUTO_REUSE))\n",
    "    x = tf.stack(xs,axis=1)\n",
    "    x = tl.layers.InputLayer(x, name='lstm2_input')\n",
    "    x = tl.layers.DynamicRNNLayer(layer=x, cell_fn = tf.contrib.rnn.BasicLSTMCell, n_hidden = unit,\\\n",
    "                                  return_last = False, return_seq_2d = False, name = 'lstm_2').outputs\n",
    "    x = tf.reshape(x, (-1,timesteps,unit))\n",
    "    # f: weights (batch_size, timestep , n_class)\n",
    "    x = tf.matmul(f, x, transpose_a=True) # n_class * units\n",
    "    x = tf.unstack(x, axis=1) # len(x)=n_class\n",
    "    xs = []\n",
    "    for x_i in x:\n",
    "        xs.append(layers.Dense(1)(x_i))\n",
    "    x = tf.stack(xs,axis=1)\n",
    "    x = tf.reshape(x, (-1,n_class))\n",
    "    x = tf.nn.softmax(x)\n",
    "\n",
    "    return x\n",
    "\n",
    "# def LSTM_model(inputs, input_dim=512, output_dim=256, unit=128, n_class=5):\n",
    "#     x = tf.unstack(inputs, axis=1) # len(x)=timestaps\n",
    "#     batch_norm_params = {\n",
    "#         # Decay for the moving averages.\n",
    "#         'decay': 0.99,\n",
    "#         # epsilon to prevent 0s in variance.\n",
    "#         'epsilon': 0.001,\n",
    "#         # force in-place updates of mean and variance estimates\n",
    "#         'updates_collections': None,\n",
    "#         # Moving averages ends up in the trainable variables collection\n",
    "#         'variables_collections': [ tf.GraphKeys.TRAINABLE_VARIABLES ],\n",
    "#     }\n",
    "#     xs = []\n",
    "#     for x_i in x:\n",
    "#         xs.append(tf.contrib.layers.fully_connected(x_i, output_dim, activation_fn=tf.nn.relu,\\\n",
    "#                                             normalizer_fn=slim.batch_norm, \\\n",
    "#                                             normalizer_params=batch_norm_params, \\\n",
    "#                                             scope=\"fc256\", reuse=tf.AUTO_REUSE))\n",
    "#     x = tf.stack(xs,axis=1)\n",
    "#     x = tl.layers.InputLayer(x, name='input_for_lstm')\n",
    "#     x = tl.layers.DynamicRNNLayer(layer=x, cell_fn = tf.contrib.rnn.BasicLSTMCell, n_hidden = unit,\\\n",
    "#                                   return_last = True, return_seq_2d = True, name = 'lstm3').outputs\n",
    "#     x = layers.Dense(n_class,activation='softmax')(x)\n",
    "    \n",
    "#     return x\n",
    "    \n",
    "dim=512\n",
    "timesteps=141\n",
    "n_class=5\n",
    "sess=tf.InteractiveSession()\n",
    "inputs = tf.placeholder(tf.float32, [None, timesteps, dim])\n",
    "print('input shape:',inputs.shape)\n",
    "y = tf.placeholder(tf.int64, [None, 1])\n",
    "y_onehot = tf.reshape(tf.one_hot(y, n_class),(-1,n_class))\n",
    "y_onehot_stacked = tf.reshape(tf.stack([y_onehot]*timesteps,axis=1),(-1,timesteps,n_class))\n",
    "\n",
    "# model & loss\n",
    "f_weights=weights_model(inputs, timesteps=timesteps, dim=dim, unit=128, emotion_embedding_dim=256, n_class=5)\n",
    "outputs=attention_model(inputs, f_weights, timesteps=timesteps, dim=dim, unit=128, n_class=5)\n",
    "# lstm_outputs=LSTM_model(inputs)\n",
    "print('f_weights shape:',f_weights.shape)\n",
    "print('outputs shape:',outputs.shape)\n",
    "# print('lstm_outputs shape:',lstm_outputs.shape)\n",
    "loss1 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=f_weights, labels=y_onehot_stacked))\n",
    "loss2 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=outputs, labels=y_onehot))\n",
    "# loss3 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=lstm_outputs, labels=y_onehot))\n",
    "# tf.summary.scalar('loss', softmax_loss)\n",
    "\n",
    "# cal acc\n",
    "result=tf.argmax(outputs,1)\n",
    "ground_truth = tf.reshape(y, [-1])\n",
    "correct_prediction = tf.equal(result, ground_truth)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "# tf.summary.scalar('accuracy', accuracy)\n",
    "\n",
    "# train\n",
    "# train_step = tf.train.AdamOptimizer(lr).minimize(softmax_loss)\n",
    "# learning_rate=tf.placeholder(tf.float32)\n",
    "update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "with tf.control_dependencies(update_ops):\n",
    "#     train_op3 = tf.train.AdamOptimizer(0.001).minimize(loss3)\n",
    "    # train_op1 = tf.train.MomentumOptimizer(learning_rate1, 0.9).minimize(loss1)\n",
    "    # train_op2 = tf.train.MomentumOptimizer(learning_rate2, 0.9).minimize(loss2)\n",
    "    train_op1 = tf.train.AdamOptimizer(0.0005).minimize(loss1)\n",
    "    train_op2 = tf.train.AdamOptimizer(0.0005).minimize(loss2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (68,)\n",
      "y_train: (68,)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "# # CASME2\n",
    "\n",
    "# file_head='CASME2/CASME2_vgg16_feature/'\n",
    "# def cat(x,file_head):\n",
    "#     if x[0]<10:\n",
    "#         return file_head+'sub0'+str(x[0])+'/'+x[1]+'.csv'\n",
    "#     else:\n",
    "#         return file_head+'sub'+str(x[0])+'/'+x[1]+'.csv'\n",
    "\n",
    "# label_path='CASME2-ObjectiveClasses.xlsx'\n",
    "# data_labels=pd.read_excel(label_path)\n",
    "# data_labels.head()\n",
    "# name_list=data_labels[['Subject','Filename']].values\n",
    "# label_list=data_labels['Objective Class'].values\n",
    "# name_list=np.array(list(map(lambda x:cat(x,file_head),name_list)))\n",
    "# index_useful=np.where(label_list<=5)[0] # 去除label6 label7\n",
    "# label_list=label_list[index_useful]\n",
    "# name_list=name_list[index_useful]\n",
    "# label_list=label_list-1\n",
    "# X_train=name_list\n",
    "# y_train=label_list\n",
    "# print('X_train:',X_train.shape)\n",
    "# print('y_train:',y_train.shape)\n",
    "\n",
    "# SAMM\n",
    "data_path='SAMM_vggface_vgg16_feature/All/'\n",
    "data_labels=os.listdir(data_path)\n",
    "#去除label6 label7\n",
    "data_labels.remove('6')\n",
    "data_labels.remove('7')\n",
    "\n",
    "people={}\n",
    "X_train=[]\n",
    "y_train=[]\n",
    "for l in data_labels:\n",
    "    list_temp=os.listdir(data_path+l+'/')\n",
    "    for j in list_temp:\n",
    "        X_train.append(data_path+l+'/'+j)\n",
    "        y_train.append(int(l)-1)\n",
    "        if j.split('_')[0] not in people:\n",
    "            people[j.split('_')[0]]=[int(l)]\n",
    "        elif int(l) not in people[j.split('_')[0]]:\n",
    "            people[j.split('_')[0]].append(int(l))          \n",
    "X_train=np.array(X_train)\n",
    "y_train=np.array(y_train)\n",
    "print('X_train:',X_train.shape)\n",
    "print('y_train:',y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train for f_weights...\n",
      "[epoch:1/300, steps:8/8] -loss: 1.5856      \n",
      "\n",
      "[epoch:2/300, steps:8/8] -loss: 1.4999      \n",
      "\n",
      "[epoch:3/300, steps:8/8] -loss: 1.4749      \n",
      "\n",
      "[epoch:4/300, steps:8/8] -loss: 1.4295      \n",
      "\n",
      "[epoch:5/300, steps:8/8] -loss: 1.3779      \n",
      "\n",
      "[epoch:6/300, steps:8/8] -loss: 1.3458      \n",
      "\n",
      "[epoch:7/300, steps:8/8] -loss: 1.3198      \n",
      "\n",
      "[epoch:8/300, steps:8/8] -loss: 1.3317      \n",
      "\n",
      "[epoch:9/300, steps:8/8] -loss: 1.2894      \n",
      "\n",
      "[epoch:10/300, steps:8/8] -loss: 1.3855      \n",
      "\n",
      "[epoch:11/300, steps:8/8] -loss: 1.4014      \n",
      "\n",
      "[epoch:12/300, steps:8/8] -loss: 1.2994      \n",
      "\n",
      "[epoch:13/300, steps:8/8] -loss: 1.2839      \n",
      "\n",
      "[epoch:14/300, steps:8/8] -loss: 1.2943      \n",
      "\n",
      "[epoch:15/300, steps:8/8] -loss: 1.3090      \n",
      "\n",
      "[epoch:16/300, steps:8/8] -loss: 1.3021      \n",
      "\n",
      "[epoch:17/300, steps:8/8] -loss: 1.2491      \n",
      "\n",
      "[epoch:18/300, steps:8/8] -loss: 1.2732      \n",
      "\n",
      "[epoch:19/300, steps:8/8] -loss: 1.2566      \n",
      "\n",
      "[epoch:20/300, steps:8/8] -loss: 1.2648      \n",
      "\n",
      "[epoch:21/300, steps:8/8] -loss: 1.2515      \n",
      "\n",
      "[epoch:22/300, steps:8/8] -loss: 1.2523      \n",
      "\n",
      "[epoch:23/300, steps:8/8] -loss: 1.2584      \n",
      "\n",
      "[epoch:24/300, steps:8/8] -loss: 1.2732      \n",
      "\n",
      "[epoch:25/300, steps:8/8] -loss: 1.2254      \n",
      "\n",
      "[epoch:26/300, steps:8/8] -loss: 1.2214      \n",
      "\n",
      "[epoch:27/300, steps:8/8] -loss: 1.2017      \n",
      "\n",
      "[epoch:28/300, steps:8/8] -loss: 1.1633      \n",
      "\n",
      "[epoch:29/300, steps:8/8] -loss: 1.2671      \n",
      "\n",
      "[epoch:30/300, steps:8/8] -loss: 1.2410      \n",
      "\n",
      "[epoch:31/300, steps:8/8] -loss: 1.2453      \n",
      "\n",
      "[epoch:32/300, steps:8/8] -loss: 1.2116      \n",
      "\n",
      "[epoch:33/300, steps:8/8] -loss: 1.1869      \n",
      "\n",
      "[epoch:34/300, steps:8/8] -loss: 1.1798      \n",
      "\n",
      "[epoch:35/300, steps:8/8] -loss: 1.1796      \n",
      "\n",
      "[epoch:36/300, steps:8/8] -loss: 1.1872      \n",
      "\n",
      "[epoch:37/300, steps:8/8] -loss: 1.2010      \n",
      "\n",
      "[epoch:38/300, steps:8/8] -loss: 1.1657      \n",
      "\n",
      "[epoch:39/300, steps:8/8] -loss: 1.2106      \n",
      "\n",
      "[epoch:40/300, steps:8/8] -loss: 1.2081      \n",
      "\n",
      "[epoch:41/300, steps:8/8] -loss: 1.2223      \n",
      "\n",
      "[epoch:42/300, steps:8/8] -loss: 1.2291      \n",
      "\n",
      "[epoch:43/300, steps:8/8] -loss: 1.2286      \n",
      "\n",
      "[epoch:44/300, steps:8/8] -loss: 1.2691      \n",
      "\n",
      "[epoch:45/300, steps:8/8] -loss: 1.2338      \n",
      "\n",
      "[epoch:46/300, steps:8/8] -loss: 1.2457      \n",
      "\n",
      "[epoch:47/300, steps:8/8] -loss: 1.2104      \n",
      "\n",
      "[epoch:48/300, steps:8/8] -loss: 1.2499      \n",
      "\n",
      "[epoch:49/300, steps:8/8] -loss: 1.2012      \n",
      "\n",
      "[epoch:50/300, steps:8/8] -loss: 1.2541      \n",
      "\n",
      "[epoch:51/300, steps:8/8] -loss: 1.2900      \n",
      "\n",
      "[epoch:52/300, steps:8/8] -loss: 1.2494      \n",
      "\n",
      "[epoch:53/300, steps:8/8] -loss: 1.2272      \n",
      "\n",
      "[epoch:54/300, steps:8/8] -loss: 1.1820      \n",
      "\n",
      "[epoch:55/300, steps:8/8] -loss: 1.3258      \n",
      "\n",
      "[epoch:56/300, steps:8/8] -loss: 1.3520      \n",
      "\n",
      "[epoch:57/300, steps:8/8] -loss: 1.3095      \n",
      "\n",
      "[epoch:58/300, steps:8/8] -loss: 1.2912      \n",
      "\n",
      "[epoch:59/300, steps:8/8] -loss: 1.2696      \n",
      "\n",
      "[epoch:60/300, steps:8/8] -loss: 1.2405      \n",
      "\n",
      "[epoch:61/300, steps:8/8] -loss: 1.2323      \n",
      "\n",
      "[epoch:62/300, steps:8/8] -loss: 1.2553      \n",
      "\n",
      "[epoch:63/300, steps:8/8] -loss: 1.2262      \n",
      "\n",
      "[epoch:64/300, steps:8/8] -loss: 1.2431      \n",
      "\n",
      "[epoch:65/300, steps:8/8] -loss: 1.2134      \n",
      "\n",
      "[epoch:66/300, steps:8/8] -loss: 1.2509      \n",
      "\n",
      "[epoch:67/300, steps:8/8] -loss: 1.2176      \n",
      "\n",
      "[epoch:68/300, steps:8/8] -loss: 1.1612      \n",
      "\n",
      "[epoch:69/300, steps:8/8] -loss: 1.2293      \n",
      "\n",
      "[epoch:70/300, steps:8/8] -loss: 1.1905      \n",
      "\n",
      "[epoch:71/300, steps:8/8] -loss: 1.1763      \n",
      "\n",
      "[epoch:72/300, steps:8/8] -loss: 1.1794      \n",
      "\n",
      "[epoch:73/300, steps:8/8] -loss: 1.1994      \n",
      "\n",
      "[epoch:74/300, steps:8/8] -loss: 1.1833      \n",
      "\n",
      "[epoch:75/300, steps:8/8] -loss: 1.1344      \n",
      "\n",
      "[epoch:76/300, steps:8/8] -loss: 1.1460      \n",
      "\n",
      "[epoch:77/300, steps:8/8] -loss: 1.1768      \n",
      "\n",
      "[epoch:78/300, steps:8/8] -loss: 1.1270      \n",
      "\n",
      "[epoch:79/300, steps:8/8] -loss: 1.1349      \n",
      "\n",
      "[epoch:80/300, steps:8/8] -loss: 1.1475      \n",
      "\n",
      "[epoch:81/300, steps:8/8] -loss: 1.1257      \n",
      "\n",
      "[epoch:82/300, steps:8/8] -loss: 1.1333      \n",
      "\n",
      "[epoch:83/300, steps:8/8] -loss: 1.1536      \n",
      "\n",
      "[epoch:84/300, steps:8/8] -loss: 1.1703      \n",
      "\n",
      "[epoch:85/300, steps:8/8] -loss: 1.0660      \n",
      "\n",
      "[epoch:86/300, steps:8/8] -loss: 1.1501      \n",
      "\n",
      "[epoch:87/300, steps:8/8] -loss: 1.1132      \n",
      "\n",
      "[epoch:88/300, steps:8/8] -loss: 1.1329      \n",
      "\n",
      "[epoch:89/300, steps:8/8] -loss: 1.1822      \n",
      "\n",
      "[epoch:90/300, steps:8/8] -loss: 1.1789      \n",
      "\n",
      "[epoch:91/300, steps:8/8] -loss: 1.1847      \n",
      "\n",
      "[epoch:92/300, steps:8/8] -loss: 1.1311      \n",
      "\n",
      "[epoch:93/300, steps:8/8] -loss: 1.1762      \n",
      "\n",
      "[epoch:94/300, steps:8/8] -loss: 1.2037      \n",
      "\n",
      "[epoch:95/300, steps:8/8] -loss: 1.1723      \n",
      "\n",
      "[epoch:96/300, steps:8/8] -loss: 1.1557      \n",
      "\n",
      "[epoch:97/300, steps:8/8] -loss: 1.1399      \n",
      "\n",
      "[epoch:98/300, steps:8/8] -loss: 1.1294      \n",
      "\n",
      "[epoch:99/300, steps:8/8] -loss: 1.1147      \n",
      "\n",
      "[epoch:100/300, steps:8/8] -loss: 1.1270      \n",
      "\n",
      "[epoch:101/300, steps:8/8] -loss: 1.1282      \n",
      "\n",
      "[epoch:102/300, steps:8/8] -loss: 1.1379      \n",
      "\n",
      "[epoch:103/300, steps:8/8] -loss: 1.1165      \n",
      "\n",
      "[epoch:104/300, steps:8/8] -loss: 1.1395      \n",
      "\n",
      "[epoch:105/300, steps:8/8] -loss: 1.1054      \n",
      "\n",
      "[epoch:106/300, steps:8/8] -loss: 1.1431      \n",
      "\n",
      "[epoch:107/300, steps:8/8] -loss: 1.1436      \n",
      "\n",
      "[epoch:108/300, steps:8/8] -loss: 1.1439      \n",
      "\n",
      "[epoch:109/300, steps:8/8] -loss: 1.1473      \n",
      "\n",
      "[epoch:110/300, steps:8/8] -loss: 1.0529      \n",
      "\n",
      "[epoch:111/300, steps:8/8] -loss: 1.0898      \n",
      "\n",
      "[epoch:112/300, steps:8/8] -loss: 1.1619      \n",
      "\n",
      "[epoch:113/300, steps:8/8] -loss: 1.1012      \n",
      "\n",
      "[epoch:114/300, steps:8/8] -loss: 1.1476      \n",
      "\n",
      "[epoch:115/300, steps:8/8] -loss: 1.1194      \n",
      "\n",
      "[epoch:116/300, steps:8/8] -loss: 1.0980      \n",
      "\n",
      "[epoch:117/300, steps:8/8] -loss: 1.1674      \n",
      "\n",
      "[epoch:118/300, steps:8/8] -loss: 1.1618      \n",
      "\n",
      "[epoch:119/300, steps:8/8] -loss: 1.1484      \n",
      "\n",
      "[epoch:120/300, steps:8/8] -loss: 1.1955      \n",
      "\n",
      "[epoch:121/300, steps:8/8] -loss: 1.2027      \n",
      "\n",
      "[epoch:122/300, steps:8/8] -loss: 1.1981      \n",
      "\n",
      "[epoch:123/300, steps:8/8] -loss: 1.2068      \n",
      "\n",
      "[epoch:124/300, steps:8/8] -loss: 1.1090      \n",
      "\n",
      "[epoch:125/300, steps:8/8] -loss: 1.2308      \n",
      "\n",
      "[epoch:126/300, steps:8/8] -loss: 1.1329      \n",
      "\n",
      "[epoch:127/300, steps:8/8] -loss: 1.1963      \n",
      "\n",
      "[epoch:128/300, steps:8/8] -loss: 1.1723      \n",
      "\n",
      "[epoch:129/300, steps:8/8] -loss: 1.1697      \n",
      "\n",
      "[epoch:130/300, steps:8/8] -loss: 1.2074      \n",
      "\n",
      "[epoch:131/300, steps:8/8] -loss: 1.1946      \n",
      "\n",
      "[epoch:132/300, steps:8/8] -loss: 1.1754      \n",
      "\n",
      "[epoch:133/300, steps:8/8] -loss: 1.1135      \n",
      "\n",
      "[epoch:134/300, steps:8/8] -loss: 1.1407      \n",
      "\n",
      "[epoch:135/300, steps:8/8] -loss: 1.1037      \n",
      "\n",
      "[epoch:136/300, steps:8/8] -loss: 1.1226      \n",
      "\n",
      "[epoch:137/300, steps:8/8] -loss: 1.0874      \n",
      "\n",
      "[epoch:138/300, steps:8/8] -loss: 1.0705      \n",
      "\n",
      "[epoch:139/300, steps:8/8] -loss: 1.1208      \n",
      "\n",
      "[epoch:140/300, steps:8/8] -loss: 1.1257      \n",
      "\n",
      "[epoch:141/300, steps:8/8] -loss: 1.1010      \n",
      "\n",
      "[epoch:142/300, steps:8/8] -loss: 1.1512      \n",
      "\n",
      "[epoch:143/300, steps:8/8] -loss: 1.0958      \n",
      "\n",
      "[epoch:144/300, steps:8/8] -loss: 1.0848      \n",
      "\n",
      "[epoch:145/300, steps:8/8] -loss: 1.1427      \n",
      "\n",
      "[epoch:146/300, steps:8/8] -loss: 1.1000      \n",
      "\n",
      "[epoch:147/300, steps:8/8] -loss: 1.0904      \n",
      "\n",
      "[epoch:148/300, steps:8/8] -loss: 1.1168      \n",
      "\n",
      "[epoch:149/300, steps:8/8] -loss: 1.0950      \n",
      "\n",
      "[epoch:150/300, steps:8/8] -loss: 1.1215      \n",
      "\n",
      "[epoch:151/300, steps:8/8] -loss: 1.1307      \n",
      "\n",
      "[epoch:152/300, steps:8/8] -loss: 1.1176      \n",
      "\n",
      "[epoch:153/300, steps:8/8] -loss: 1.1059      \n",
      "\n",
      "[epoch:154/300, steps:8/8] -loss: 1.0958      \n",
      "\n",
      "[epoch:155/300, steps:8/8] -loss: 1.1032      \n",
      "\n",
      "[epoch:156/300, steps:8/8] -loss: 1.0680      \n",
      "\n",
      "[epoch:157/300, steps:8/8] -loss: 1.1021      \n",
      "\n",
      "[epoch:158/300, steps:8/8] -loss: 1.1039      \n",
      "\n",
      "[epoch:159/300, steps:8/8] -loss: 1.1213      \n",
      "\n",
      "[epoch:160/300, steps:8/8] -loss: 1.0821      \n",
      "\n",
      "[epoch:161/300, steps:8/8] -loss: 1.1008      \n",
      "\n",
      "[epoch:162/300, steps:8/8] -loss: 1.0527      \n",
      "\n",
      "[epoch:163/300, steps:8/8] -loss: 1.0738      \n",
      "\n",
      "[epoch:164/300, steps:8/8] -loss: 1.0747      \n",
      "\n",
      "[epoch:165/300, steps:8/8] -loss: 1.0912      \n",
      "\n",
      "[epoch:166/300, steps:8/8] -loss: 1.0879      \n",
      "\n",
      "[epoch:167/300, steps:8/8] -loss: 1.0925      \n",
      "\n",
      "[epoch:168/300, steps:8/8] -loss: 1.1614      \n",
      "\n",
      "[epoch:169/300, steps:8/8] -loss: 1.0784      \n",
      "\n",
      "[epoch:170/300, steps:8/8] -loss: 1.0974      \n",
      "\n",
      "[epoch:171/300, steps:8/8] -loss: 1.0993      \n",
      "\n",
      "[epoch:172/300, steps:8/8] -loss: 1.0675      \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch:173/300, steps:8/8] -loss: 1.0491      \n",
      "\n",
      "[epoch:174/300, steps:8/8] -loss: 1.1105      \n",
      "\n",
      "[epoch:175/300, steps:8/8] -loss: 1.0790      \n",
      "\n",
      "[epoch:176/300, steps:8/8] -loss: 1.0725      \n",
      "\n",
      "[epoch:177/300, steps:8/8] -loss: 1.0936      \n",
      "\n",
      "[epoch:178/300, steps:8/8] -loss: 1.1038      \n",
      "\n",
      "[epoch:179/300, steps:8/8] -loss: 1.0792      \n",
      "\n",
      "[epoch:180/300, steps:8/8] -loss: 1.0828      \n",
      "\n",
      "[epoch:181/300, steps:8/8] -loss: 1.1016      \n",
      "\n",
      "[epoch:182/300, steps:8/8] -loss: 1.0577      \n",
      "\n",
      "[epoch:183/300, steps:8/8] -loss: 1.0689      \n",
      "\n",
      "[epoch:184/300, steps:8/8] -loss: 1.1350      \n",
      "\n",
      "[epoch:185/300, steps:8/8] -loss: 1.0743      \n",
      "\n",
      "[epoch:186/300, steps:8/8] -loss: 1.0362      \n",
      "\n",
      "[epoch:187/300, steps:8/8] -loss: 1.1045      \n",
      "\n",
      "[epoch:188/300, steps:8/8] -loss: 1.0812      \n",
      "\n",
      "[epoch:189/300, steps:8/8] -loss: 1.0913      \n",
      "\n",
      "[epoch:190/300, steps:8/8] -loss: 1.0982      \n",
      "\n",
      "[epoch:191/300, steps:8/8] -loss: 1.1457      \n",
      "\n",
      "[epoch:192/300, steps:8/8] -loss: 1.1681      \n",
      "\n",
      "[epoch:193/300, steps:8/8] -loss: 1.1271      \n",
      "\n",
      "[epoch:194/300, steps:8/8] -loss: 1.1346      \n",
      "\n",
      "[epoch:195/300, steps:8/8] -loss: 1.0995      \n",
      "\n",
      "[epoch:196/300, steps:8/8] -loss: 1.0702      \n",
      "\n",
      "[epoch:197/300, steps:8/8] -loss: 1.0865      \n",
      "\n",
      "[epoch:198/300, steps:8/8] -loss: 1.1031      \n",
      "\n",
      "[epoch:199/300, steps:8/8] -loss: 1.0581      \n",
      "\n",
      "[epoch:200/300, steps:8/8] -loss: 1.0479      \n",
      "\n",
      "[epoch:201/300, steps:8/8] -loss: 1.0778      \n",
      "\n",
      "[epoch:202/300, steps:8/8] -loss: 1.0947      \n",
      "\n",
      "[epoch:203/300, steps:8/8] -loss: 1.1406      \n",
      "\n",
      "[epoch:204/300, steps:8/8] -loss: 1.1282      \n",
      "\n",
      "[epoch:205/300, steps:8/8] -loss: 1.1045      \n",
      "\n",
      "[epoch:206/300, steps:8/8] -loss: 1.0695      \n",
      "\n",
      "[epoch:207/300, steps:8/8] -loss: 1.0697      \n",
      "\n",
      "[epoch:208/300, steps:8/8] -loss: 1.0718      \n",
      "\n",
      "[epoch:209/300, steps:8/8] -loss: 1.0885      \n",
      "\n",
      "[epoch:210/300, steps:8/8] -loss: 1.0786      \n",
      "\n",
      "[epoch:211/300, steps:8/8] -loss: 1.0916      \n",
      "\n",
      "[epoch:212/300, steps:8/8] -loss: 1.0866      \n",
      "\n",
      "[epoch:213/300, steps:8/8] -loss: 1.0645      \n",
      "\n",
      "[epoch:214/300, steps:8/8] -loss: 1.0491      \n",
      "\n",
      "[epoch:215/300, steps:8/8] -loss: 1.0528      \n",
      "\n",
      "[epoch:216/300, steps:8/8] -loss: 1.0599      \n",
      "\n",
      "[epoch:217/300, steps:8/8] -loss: 1.0711      \n",
      "\n",
      "[epoch:218/300, steps:8/8] -loss: 1.0836      \n",
      "\n",
      "[epoch:219/300, steps:8/8] -loss: 1.0638      \n",
      "\n",
      "[epoch:220/300, steps:8/8] -loss: 1.0521      \n",
      "\n",
      "[epoch:221/300, steps:8/8] -loss: 1.1121      \n",
      "\n",
      "[epoch:222/300, steps:8/8] -loss: 1.0565      \n",
      "\n",
      "[epoch:223/300, steps:8/8] -loss: 1.0610      \n",
      "\n",
      "[epoch:224/300, steps:8/8] -loss: 1.0739      \n",
      "\n",
      "[epoch:225/300, steps:8/8] -loss: 1.0137      \n",
      "\n",
      "[epoch:226/300, steps:8/8] -loss: 1.0432      \n",
      "\n",
      "[epoch:227/300, steps:8/8] -loss: 1.0705      \n",
      "\n",
      "[epoch:228/300, steps:8/8] -loss: 1.0469      \n",
      "\n",
      "[epoch:229/300, steps:8/8] -loss: 1.1378      \n",
      "\n",
      "[epoch:230/300, steps:8/8] -loss: 1.0958      \n",
      "\n",
      "[epoch:231/300, steps:8/8] -loss: 1.0534      \n",
      "\n",
      "[epoch:232/300, steps:8/8] -loss: 1.0516      \n",
      "\n",
      "[epoch:233/300, steps:8/8] -loss: 1.0694      \n",
      "\n",
      "[epoch:234/300, steps:8/8] -loss: 1.0954      \n",
      "\n",
      "[epoch:235/300, steps:8/8] -loss: 1.0756      \n",
      "\n",
      "[epoch:236/300, steps:8/8] -loss: 1.0510      \n",
      "\n",
      "[epoch:237/300, steps:8/8] -loss: 1.0406      \n",
      "\n",
      "[epoch:238/300, steps:8/8] -loss: 1.0380      \n",
      "\n",
      "[epoch:239/300, steps:8/8] -loss: 1.0495      \n",
      "\n",
      "[epoch:240/300, steps:8/8] -loss: 1.0553      \n",
      "\n",
      "[epoch:241/300, steps:8/8] -loss: 1.1006      \n",
      "\n",
      "[epoch:242/300, steps:8/8] -loss: 1.1215      \n",
      "\n",
      "[epoch:243/300, steps:8/8] -loss: 1.1146      \n",
      "\n",
      "[epoch:244/300, steps:8/8] -loss: 1.0809      \n",
      "\n",
      "[epoch:245/300, steps:8/8] -loss: 1.0609      \n",
      "\n",
      "[epoch:246/300, steps:8/8] -loss: 1.1169      \n",
      "\n",
      "[epoch:247/300, steps:8/8] -loss: 1.0805      \n",
      "\n",
      "[epoch:248/300, steps:8/8] -loss: 1.0871      \n",
      "\n",
      "[epoch:249/300, steps:8/8] -loss: 1.0877      \n",
      "\n",
      "[epoch:250/300, steps:8/8] -loss: 1.1167      \n",
      "\n",
      "[epoch:251/300, steps:8/8] -loss: 1.0958      \n",
      "\n",
      "[epoch:252/300, steps:8/8] -loss: 1.0731      \n",
      "\n",
      "[epoch:253/300, steps:8/8] -loss: 1.0751      \n",
      "\n",
      "[epoch:254/300, steps:8/8] -loss: 1.0835      \n",
      "\n",
      "[epoch:255/300, steps:8/8] -loss: 1.0802      \n",
      "\n",
      "[epoch:256/300, steps:8/8] -loss: 1.0412      \n",
      "\n",
      "[epoch:257/300, steps:8/8] -loss: 1.0611      \n",
      "\n",
      "[epoch:258/300, steps:8/8] -loss: 1.0456      \n",
      "\n",
      "[epoch:259/300, steps:8/8] -loss: 1.0647      \n",
      "\n",
      "[epoch:260/300, steps:8/8] -loss: 1.0622      \n",
      "\n",
      "[epoch:261/300, steps:8/8] -loss: 1.0531      \n",
      "\n",
      "[epoch:262/300, steps:8/8] -loss: 1.0540      \n",
      "\n",
      "[epoch:263/300, steps:8/8] -loss: 1.0838      \n",
      "\n",
      "[epoch:264/300, steps:8/8] -loss: 1.0682      \n",
      "\n",
      "[epoch:265/300, steps:8/8] -loss: 1.0482      \n",
      "\n",
      "[epoch:266/300, steps:8/8] -loss: 1.0683      \n",
      "\n",
      "[epoch:267/300, steps:8/8] -loss: 1.0763      \n",
      "\n",
      "[epoch:268/300, steps:8/8] -loss: 1.0048      \n",
      "\n",
      "[epoch:269/300, steps:8/8] -loss: 1.0175      \n",
      "\n",
      "[epoch:270/300, steps:8/8] -loss: 1.0505      \n",
      "\n",
      "[epoch:271/300, steps:8/8] -loss: 1.0278      \n",
      "\n",
      "[epoch:272/300, steps:8/8] -loss: 1.0603      \n",
      "\n",
      "[epoch:273/300, steps:8/8] -loss: 1.0768      \n",
      "\n",
      "[epoch:274/300, steps:8/8] -loss: 1.1709      \n",
      "\n",
      "[epoch:275/300, steps:8/8] -loss: 1.1609      \n",
      "\n",
      "[epoch:276/300, steps:8/8] -loss: 1.1340      \n",
      "\n",
      "[epoch:277/300, steps:8/8] -loss: 1.1207      \n",
      "\n",
      "[epoch:278/300, steps:8/8] -loss: 1.1229      \n",
      "\n",
      "[epoch:279/300, steps:8/8] -loss: 1.1115      \n",
      "\n",
      "[epoch:280/300, steps:8/8] -loss: 1.1238      \n",
      "\n",
      "[epoch:281/300, steps:8/8] -loss: 1.1031      \n",
      "\n",
      "[epoch:282/300, steps:8/8] -loss: 1.0750      \n",
      "\n",
      "[epoch:283/300, steps:8/8] -loss: 1.1027      \n",
      "\n",
      "[epoch:284/300, steps:8/8] -loss: 1.1159      \n",
      "\n",
      "[epoch:285/300, steps:8/8] -loss: 1.1177      \n",
      "\n",
      "[epoch:286/300, steps:8/8] -loss: 1.1457      \n",
      "\n",
      "[epoch:287/300, steps:8/8] -loss: 1.1627      \n",
      "\n",
      "[epoch:288/300, steps:8/8] -loss: 1.1884      \n",
      "\n",
      "[epoch:289/300, steps:8/8] -loss: 1.1047      \n",
      "\n",
      "[epoch:290/300, steps:8/8] -loss: 1.0741      \n",
      "\n",
      "[epoch:291/300, steps:8/8] -loss: 1.0662      \n",
      "\n",
      "[epoch:292/300, steps:8/8] -loss: 1.0738      \n",
      "\n",
      "[epoch:293/300, steps:8/8] -loss: 1.0904      \n",
      "\n",
      "[epoch:294/300, steps:8/8] -loss: 1.0886      \n",
      "\n",
      "[epoch:295/300, steps:8/8] -loss: 1.0559      \n",
      "\n",
      "[epoch:296/300, steps:8/8] -loss: 1.0757      \n",
      "\n",
      "[epoch:297/300, steps:8/8] -loss: 1.0670      \n",
      "\n",
      "[epoch:298/300, steps:8/8] -loss: 1.0625      \n",
      "\n",
      "[epoch:299/300, steps:8/8] -loss: 1.0972      \n",
      "\n",
      "[epoch:300/300, steps:8/8] -loss: 1.0907      \n",
      "\n",
      "train for the attention model...\n",
      "[epoch:1/300, steps:8/8] -loss: 1.4464 - acc: 43.75%      \n",
      "\n",
      "[epoch:2/300, steps:8/8] -loss: 1.3680 - acc: 51.56%      \n",
      "\n",
      "[epoch:3/300, steps:8/8] -loss: 1.3663 - acc: 51.56%      \n",
      "\n",
      "[epoch:4/300, steps:8/8] -loss: 1.3758 - acc: 51.56%      \n",
      "\n",
      "[epoch:5/300, steps:8/8] -loss: 1.3421 - acc: 56.25%       \n",
      "\n",
      "[epoch:6/300, steps:8/8] -loss: 1.3472 - acc: 54.69%      \n",
      "\n",
      "[epoch:7/300, steps:8/8] -loss: 1.3403 - acc: 54.69%      \n",
      "\n",
      "[epoch:8/300, steps:8/8] -loss: 1.4071 - acc: 50.00%      \n",
      "\n",
      "[epoch:9/300, steps:8/8] -loss: 1.4457 - acc: 45.31%      \n",
      "\n",
      "[epoch:10/300, steps:8/8] -loss: 1.4929 - acc: 40.62%      \n",
      "\n",
      "[epoch:11/300, steps:8/8] -loss: 1.4853 - acc: 39.06%      \n",
      "\n",
      "[epoch:12/300, steps:8/8] -loss: 1.5000 - acc: 40.62%      \n",
      "\n",
      "[epoch:13/300, steps:8/8] -loss: 1.5073 - acc: 37.50%      \n",
      "\n",
      "[epoch:14/300, steps:8/8] -loss: 1.4591 - acc: 42.19%      \n",
      "\n",
      "[epoch:15/300, steps:8/8] -loss: 1.4517 - acc: 43.75%      \n",
      "\n",
      "[epoch:16/300, steps:8/8] -loss: 1.4782 - acc: 40.62%      \n",
      "\n",
      "[epoch:17/300, steps:8/8] -loss: 1.4179 - acc: 46.88%      \n",
      "\n",
      "[epoch:18/300, steps:8/8] -loss: 1.4751 - acc: 40.62%      \n",
      "\n",
      "[epoch:19/300, steps:8/8] -loss: 1.5318 - acc: 35.94%      \n",
      "\n",
      "[epoch:20/300, steps:8/8] -loss: 1.4764 - acc: 42.19%      \n",
      "\n",
      "[epoch:21/300, steps:8/8] -loss: 1.4514 - acc: 43.75%      \n",
      "\n",
      "[epoch:22/300, steps:8/8] -loss: 1.4237 - acc: 46.88%      \n",
      "\n",
      "[epoch:23/300, steps:8/8] -loss: 1.4289 - acc: 45.31%      \n",
      "\n",
      "[epoch:24/300, steps:8/8] -loss: 1.4166 - acc: 48.44%      \n",
      "\n",
      "[epoch:25/300, steps:8/8] -loss: 1.4274 - acc: 46.88%      \n",
      "\n",
      "[epoch:26/300, steps:8/8] -loss: 1.4942 - acc: 39.06%      \n",
      "\n",
      "[epoch:27/300, steps:8/8] -loss: 1.4676 - acc: 42.19%      \n",
      "\n",
      "[epoch:28/300, steps:8/8] -loss: 1.3929 - acc: 48.44%      \n",
      "\n",
      "[epoch:29/300, steps:8/8] -loss: 1.4619 - acc: 42.19%      \n",
      "\n",
      "[epoch:30/300, steps:8/8] -loss: 1.3894 - acc: 50.00%      \n",
      "\n",
      "[epoch:31/300, steps:8/8] -loss: 1.4037 - acc: 48.44%      \n",
      "\n",
      "[epoch:32/300, steps:8/8] -loss: 1.4826 - acc: 43.75%      \n",
      "\n",
      "[epoch:33/300, steps:8/8] -loss: 1.4600 - acc: 43.75%      \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch:34/300, steps:8/8] -loss: 1.4282 - acc: 46.88%      \n",
      "\n",
      "[epoch:35/300, steps:8/8] -loss: 1.3868 - acc: 50.00%      \n",
      "\n",
      "[epoch:36/300, steps:8/8] -loss: 1.3862 - acc: 51.56%      \n",
      "\n",
      "[epoch:37/300, steps:8/8] -loss: 1.4033 - acc: 48.44%      \n",
      "\n",
      "[epoch:38/300, steps:8/8] -loss: 1.3934 - acc: 51.56%      \n",
      "\n",
      "[epoch:39/300, steps:8/8] -loss: 1.4096 - acc: 48.44%      \n",
      "\n",
      "[epoch:40/300, steps:8/8] -loss: 1.3699 - acc: 51.56%      \n",
      "\n",
      "[epoch:41/300, steps:8/8] -loss: 1.3659 - acc: 51.56%      \n",
      "\n",
      "[epoch:42/300, steps:8/8] -loss: 1.4094 - acc: 50.00%      \n",
      "\n",
      "[epoch:43/300, steps:8/8] -loss: 1.3614 - acc: 53.12%      \n",
      "\n",
      "[epoch:44/300, steps:8/8] -loss: 1.4254 - acc: 46.88%      \n",
      "\n",
      "[epoch:45/300, steps:8/8] -loss: 1.4039 - acc: 51.56%      \n",
      "\n",
      "[epoch:46/300, steps:8/8] -loss: 1.3745 - acc: 51.56%      \n",
      "\n",
      "[epoch:47/300, steps:8/8] -loss: 1.4191 - acc: 46.88%      \n",
      "\n",
      "[epoch:48/300, steps:8/8] -loss: 1.3875 - acc: 51.56%      \n",
      "\n",
      "[epoch:49/300, steps:8/8] -loss: 1.3956 - acc: 50.00%      \n",
      "\n",
      "[epoch:50/300, steps:8/8] -loss: 1.3868 - acc: 50.00%      \n",
      "\n",
      "[epoch:51/300, steps:8/8] -loss: 1.3960 - acc: 50.00%      \n",
      "\n",
      "[epoch:52/300, steps:8/8] -loss: 1.4582 - acc: 43.75%      \n",
      "\n",
      "[epoch:53/300, steps:8/8] -loss: 1.4644 - acc: 43.75%      \n",
      "\n",
      "[epoch:54/300, steps:8/8] -loss: 1.4351 - acc: 46.88%      \n",
      "\n",
      "[epoch:55/300, steps:8/8] -loss: 1.4502 - acc: 45.31%      \n",
      "\n",
      "[epoch:56/300, steps:8/8] -loss: 1.4499 - acc: 45.31%      \n",
      "\n",
      "[epoch:57/300, steps:8/8] -loss: 1.4510 - acc: 45.31%      \n",
      "\n",
      "[epoch:58/300, steps:8/8] -loss: 1.4774 - acc: 42.19%      \n",
      "\n",
      "[epoch:59/300, steps:8/8] -loss: 1.4958 - acc: 40.62%      \n",
      "\n",
      "[epoch:60/300, steps:8/8] -loss: 1.4647 - acc: 43.75%      \n",
      "\n",
      "[epoch:61/300, steps:8/8] -loss: 1.4943 - acc: 40.62%      \n",
      "\n",
      "[epoch:62/300, steps:8/8] -loss: 1.4048 - acc: 50.00%      \n",
      "\n",
      "[epoch:63/300, steps:8/8] -loss: 1.4388 - acc: 46.88%      \n",
      "\n",
      "[epoch:64/300, steps:8/8] -loss: 1.4785 - acc: 42.19%      \n",
      "\n",
      "[epoch:65/300, steps:8/8] -loss: 1.4516 - acc: 45.31%      \n",
      "\n",
      "[epoch:66/300, steps:8/8] -loss: 1.4509 - acc: 45.31%      \n",
      "\n",
      "[epoch:67/300, steps:8/8] -loss: 1.4192 - acc: 48.44%      \n",
      "\n",
      "[epoch:68/300, steps:8/8] -loss: 1.4361 - acc: 46.88%      \n",
      "\n",
      "[epoch:69/300, steps:8/8] -loss: 1.4043 - acc: 50.00%      \n",
      "\n",
      "[epoch:70/300, steps:8/8] -loss: 1.4045 - acc: 50.00%      \n",
      "\n",
      "[epoch:71/300, steps:8/8] -loss: 1.4422 - acc: 46.88%      \n",
      "\n",
      "[epoch:72/300, steps:8/8] -loss: 1.4273 - acc: 46.88%      \n",
      "\n",
      "[epoch:73/300, steps:8/8] -loss: 1.4957 - acc: 40.62%      \n",
      "\n",
      "[epoch:74/300, steps:8/8] -loss: 1.4831 - acc: 42.19%      \n",
      "\n",
      "[epoch:75/300, steps:8/8] -loss: 1.4930 - acc: 40.62%      \n",
      "\n",
      "[epoch:76/300, steps:8/8] -loss: 1.4047 - acc: 50.00%      \n",
      "\n",
      "[epoch:77/300, steps:8/8] -loss: 1.4654 - acc: 43.75%      \n",
      "\n",
      "[epoch:78/300, steps:8/8] -loss: 1.4516 - acc: 45.31%      \n",
      "\n",
      "[epoch:79/300, steps:8/8] -loss: 1.4468 - acc: 45.31%      \n",
      "\n",
      "[epoch:80/300, steps:8/8] -loss: 1.4148 - acc: 48.44%      \n",
      "\n",
      "[epoch:81/300, steps:8/8] -loss: 1.4345 - acc: 46.88%      \n",
      "\n",
      "[epoch:82/300, steps:8/8] -loss: 1.4596 - acc: 43.75%      \n",
      "\n",
      "[epoch:83/300, steps:8/8] -loss: 1.5244 - acc: 37.50%      \n",
      "\n",
      "[epoch:84/300, steps:8/8] -loss: 1.5174 - acc: 37.50%      \n",
      "\n",
      "[epoch:85/300, steps:8/8] -loss: 1.4796 - acc: 42.19%      \n",
      "\n",
      "[epoch:86/300, steps:8/8] -loss: 1.4288 - acc: 46.88%      \n",
      "\n",
      "[epoch:87/300, steps:8/8] -loss: 1.4025 - acc: 50.00%      \n",
      "\n",
      "[epoch:88/300, steps:8/8] -loss: 1.4188 - acc: 48.44%      \n",
      "\n",
      "[epoch:89/300, steps:8/8] -loss: 1.4572 - acc: 43.75%      \n",
      "\n",
      "[epoch:90/300, steps:8/8] -loss: 1.4171 - acc: 48.44%      \n",
      "\n",
      "[epoch:91/300, steps:8/8] -loss: 1.4256 - acc: 46.88%      \n",
      "\n",
      "[epoch:92/300, steps:8/8] -loss: 1.4736 - acc: 42.19%      \n",
      "\n",
      "[epoch:93/300, steps:8/8] -loss: 1.4638 - acc: 43.75%      \n",
      "\n",
      "[epoch:94/300, steps:8/8] -loss: 1.4157 - acc: 48.44%      \n",
      "\n",
      "[epoch:95/300, steps:8/8] -loss: 1.3834 - acc: 51.56%      \n",
      "\n",
      "[epoch:96/300, steps:8/8] -loss: 1.4317 - acc: 46.88%      \n",
      "\n",
      "[epoch:97/300, steps:8/8] -loss: 1.4454 - acc: 45.31%      \n",
      "\n",
      "[epoch:98/300, steps:8/8] -loss: 1.4662 - acc: 43.75%      \n",
      "\n",
      "[epoch:99/300, steps:8/8] -loss: 1.4139 - acc: 48.44%      \n",
      "\n",
      "[epoch:100/300, steps:8/8] -loss: 1.3987 - acc: 50.00%      \n",
      "\n",
      "[epoch:101/300, steps:8/8] -loss: 1.3892 - acc: 51.56%      \n",
      "\n",
      "[epoch:102/300, steps:8/8] -loss: 1.3740 - acc: 51.56%      \n",
      "\n",
      "[epoch:103/300, steps:8/8] -loss: 1.4256 - acc: 46.88%      \n",
      "\n",
      "[epoch:104/300, steps:8/8] -loss: 1.3169 - acc: 57.81%      \n",
      "\n",
      "[epoch:105/300, steps:8/8] -loss: 1.3682 - acc: 53.12%      \n",
      "\n",
      "[epoch:106/300, steps:8/8] -loss: 1.4672 - acc: 43.75%      \n",
      "\n",
      "[epoch:107/300, steps:8/8] -loss: 1.4709 - acc: 43.75%      \n",
      "\n",
      "[epoch:108/300, steps:8/8] -loss: 1.4490 - acc: 45.31%      \n",
      "\n",
      "[epoch:109/300, steps:8/8] -loss: 1.4438 - acc: 45.31%      \n",
      "\n",
      "[epoch:110/300, steps:8/8] -loss: 1.4971 - acc: 40.62%      \n",
      "\n",
      "[epoch:111/300, steps:8/8] -loss: 1.4518 - acc: 45.31%      \n",
      "\n",
      "[epoch:112/300, steps:8/8] -loss: 1.4166 - acc: 48.44%      \n",
      "\n",
      "[epoch:113/300, steps:8/8] -loss: 1.4328 - acc: 46.88%      \n",
      "\n",
      "[epoch:114/300, steps:8/8] -loss: 1.3797 - acc: 53.12%      \n",
      "\n",
      "[epoch:115/300, steps:8/8] -loss: 1.4392 - acc: 46.88%      \n",
      "\n",
      "[epoch:116/300, steps:8/8] -loss: 1.3871 - acc: 51.56%      \n",
      "\n",
      "[epoch:117/300, steps:8/8] -loss: 1.3649 - acc: 53.12%      \n",
      "\n",
      "[epoch:118/300, steps:8/8] -loss: 1.3991 - acc: 50.00%      \n",
      "\n",
      "[epoch:119/300, steps:8/8] -loss: 1.4459 - acc: 45.31%      \n",
      "\n",
      "[epoch:120/300, steps:8/8] -loss: 1.4331 - acc: 46.88%      \n",
      "\n",
      "[epoch:121/300, steps:8/8] -loss: 1.4513 - acc: 45.31%      \n",
      "\n",
      "[epoch:122/300, steps:8/8] -loss: 1.4222 - acc: 48.44%      \n",
      "\n",
      "[epoch:123/300, steps:8/8] -loss: 1.3976 - acc: 50.00%      \n",
      "\n",
      "[epoch:124/300, steps:8/8] -loss: 1.4347 - acc: 46.88%      \n",
      "\n",
      "[epoch:125/300, steps:8/8] -loss: 1.3872 - acc: 51.56%      \n",
      "\n",
      "[epoch:126/300, steps:8/8] -loss: 1.4361 - acc: 46.88%      \n",
      "\n",
      "[epoch:127/300, steps:8/8] -loss: 1.3840 - acc: 51.56%      \n",
      "\n",
      "[epoch:128/300, steps:8/8] -loss: 1.3764 - acc: 53.12%      \n",
      "\n",
      "[epoch:129/300, steps:8/8] -loss: 1.4315 - acc: 46.88%      \n",
      "\n",
      "[epoch:130/300, steps:8/8] -loss: 1.4293 - acc: 46.88%      \n",
      "\n",
      "[epoch:131/300, steps:8/8] -loss: 1.4071 - acc: 48.44%      \n",
      "\n",
      "[epoch:132/300, steps:8/8] -loss: 1.4099 - acc: 50.00%      \n",
      "\n",
      "[epoch:133/300, steps:8/8] -loss: 1.4326 - acc: 46.88%      \n",
      "\n",
      "[epoch:134/300, steps:8/8] -loss: 1.4449 - acc: 45.31%      \n",
      "\n",
      "[epoch:135/300, steps:8/8] -loss: 1.4604 - acc: 43.75%      \n",
      "\n",
      "[epoch:136/300, steps:8/8] -loss: 1.4264 - acc: 48.44%      \n",
      "\n",
      "[epoch:137/300, steps:8/8] -loss: 1.4445 - acc: 45.31%      \n",
      "\n",
      "[epoch:138/300, steps:8/8] -loss: 1.3785 - acc: 53.12%      \n",
      "\n",
      "[epoch:139/300, steps:8/8] -loss: 1.4616 - acc: 43.75%      \n",
      "\n",
      "[epoch:140/300, steps:8/8] -loss: 1.4576 - acc: 45.31%      \n",
      "\n",
      "[epoch:141/300, steps:8/8] -loss: 1.4235 - acc: 48.44%      \n",
      "\n",
      "[epoch:142/300, steps:8/8] -loss: 1.4297 - acc: 46.88%      \n",
      "\n",
      "[epoch:143/300, steps:8/8] -loss: 1.4331 - acc: 48.44%      \n",
      "\n",
      "[epoch:144/300, steps:8/8] -loss: 1.4325 - acc: 46.88%      \n",
      "\n",
      "[epoch:145/300, steps:8/8] -loss: 1.3874 - acc: 51.56%      \n",
      "\n",
      "[epoch:146/300, steps:8/8] -loss: 1.4588 - acc: 43.75%      \n",
      "\n",
      "[epoch:147/300, steps:8/8] -loss: 1.4659 - acc: 43.75%      \n",
      "\n",
      "[epoch:148/300, steps:8/8] -loss: 1.4201 - acc: 48.44%      \n",
      "\n",
      "[epoch:149/300, steps:8/8] -loss: 1.4367 - acc: 46.88%      \n",
      "\n",
      "[epoch:150/300, steps:8/8] -loss: 1.4023 - acc: 50.00%      \n",
      "\n",
      "[epoch:151/300, steps:8/8] -loss: 1.4535 - acc: 45.31%      \n",
      "\n",
      "[epoch:152/300, steps:8/8] -loss: 1.4455 - acc: 45.31%      \n",
      "\n",
      "[epoch:153/300, steps:8/8] -loss: 1.4543 - acc: 45.31%      \n",
      "\n",
      "[epoch:154/300, steps:8/8] -loss: 1.4182 - acc: 48.44%      \n",
      "\n",
      "[epoch:155/300, steps:8/8] -loss: 1.4703 - acc: 43.75%      \n",
      "\n",
      "[epoch:156/300, steps:8/8] -loss: 1.4304 - acc: 48.44%      \n",
      "\n",
      "[epoch:157/300, steps:8/8] -loss: 1.4688 - acc: 43.75%      \n",
      "\n",
      "[epoch:158/300, steps:8/8] -loss: 1.4027 - acc: 50.00%      \n",
      "\n",
      "[epoch:159/300, steps:8/8] -loss: 1.3912 - acc: 51.56%      \n",
      "\n",
      "[epoch:160/300, steps:8/8] -loss: 1.4669 - acc: 43.75%      \n",
      "\n",
      "[epoch:161/300, steps:8/8] -loss: 1.4048 - acc: 50.00%      \n",
      "\n",
      "[epoch:162/300, steps:8/8] -loss: 1.4342 - acc: 46.88%      \n",
      "\n",
      "[epoch:163/300, steps:8/8] -loss: 1.4507 - acc: 45.31%      \n",
      "\n",
      "[epoch:164/300, steps:8/8] -loss: 1.4460 - acc: 45.31%      \n",
      "\n",
      "[epoch:165/300, steps:8/8] -loss: 1.4135 - acc: 48.44%      \n",
      "\n",
      "[epoch:166/300, steps:8/8] -loss: 1.4148 - acc: 48.44%      \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch:167/300, steps:8/8] -loss: 1.4297 - acc: 46.88%      \n",
      "\n",
      "[epoch:168/300, steps:8/8] -loss: 1.4158 - acc: 48.44%      \n",
      "\n",
      "[epoch:169/300, steps:8/8] -loss: 1.4304 - acc: 48.44%      \n",
      "\n",
      "[epoch:170/300, steps:8/8] -loss: 1.4510 - acc: 45.31%      \n",
      "\n",
      "[epoch:171/300, steps:8/8] -loss: 1.4823 - acc: 42.19%      \n",
      "\n",
      "[epoch:172/300, steps:8/8] -loss: 1.5428 - acc: 35.94%      \n",
      "\n",
      "[epoch:173/300, steps:8/8] -loss: 1.4647 - acc: 43.75%      \n",
      "\n",
      "[epoch:174/300, steps:8/8] -loss: 1.5067 - acc: 39.06%      \n",
      "\n",
      "[epoch:175/300, steps:8/8] -loss: 1.5431 - acc: 35.94%      \n",
      "\n",
      "[epoch:176/300, steps:8/8] -loss: 1.4952 - acc: 40.62%      \n",
      "\n",
      "[epoch:177/300, steps:8/8] -loss: 1.5706 - acc: 32.81%      \n",
      "\n",
      "[epoch:178/300, steps:8/8] -loss: 1.5084 - acc: 39.06%      \n",
      "\n",
      "[epoch:179/300, steps:8/8] -loss: 1.4962 - acc: 40.62%      \n",
      "\n",
      "[epoch:180/300, steps:8/8] -loss: 1.5579 - acc: 34.38%      \n",
      "\n",
      "[epoch:181/300, steps:8/8] -loss: 1.4327 - acc: 46.88%      \n",
      "\n",
      "[epoch:182/300, steps:8/8] -loss: 1.4526 - acc: 45.31%      \n",
      "\n",
      "[epoch:183/300, steps:8/8] -loss: 1.4336 - acc: 46.88%      \n",
      "\n",
      "[epoch:184/300, steps:8/8] -loss: 1.4635 - acc: 43.75%      \n",
      "\n",
      "[epoch:185/300, steps:8/8] -loss: 1.4974 - acc: 40.62%      \n",
      "\n",
      "[epoch:186/300, steps:8/8] -loss: 1.4206 - acc: 48.44%      \n",
      "\n",
      "[epoch:187/300, steps:8/8] -loss: 1.3998 - acc: 50.00%      \n",
      "\n",
      "[epoch:188/300, steps:8/8] -loss: 1.4473 - acc: 45.31%      \n",
      "\n",
      "[epoch:189/300, steps:8/8] -loss: 1.4622 - acc: 43.75%      \n",
      "\n",
      "[epoch:190/300, steps:8/8] -loss: 1.4623 - acc: 43.75%      \n",
      "\n",
      "[epoch:191/300, steps:8/8] -loss: 1.4497 - acc: 45.31%      \n",
      "\n",
      "[epoch:192/300, steps:8/8] -loss: 1.3989 - acc: 50.00%      \n",
      "\n",
      "[epoch:193/300, steps:8/8] -loss: 1.4637 - acc: 43.75%      \n",
      "\n",
      "[epoch:194/300, steps:8/8] -loss: 1.4332 - acc: 46.88%      \n",
      "\n",
      "[epoch:195/300, steps:8/8] -loss: 1.3992 - acc: 50.00%      \n",
      "\n",
      "[epoch:196/300, steps:8/8] -loss: 1.4536 - acc: 45.31%      \n",
      "\n",
      "[epoch:197/300, steps:8/8] -loss: 1.5015 - acc: 40.62%      \n",
      "\n",
      "[epoch:198/300, steps:8/8] -loss: 1.4783 - acc: 42.19%      \n",
      "\n",
      "[epoch:199/300, steps:8/8] -loss: 1.5044 - acc: 39.06%      \n",
      "\n",
      "[epoch:200/300, steps:8/8] -loss: 1.4467 - acc: 45.31%      \n",
      "\n",
      "[epoch:201/300, steps:8/8] -loss: 1.4138 - acc: 48.44%      \n",
      "\n",
      "[epoch:202/300, steps:8/8] -loss: 1.4158 - acc: 48.44%      \n",
      "\n",
      "[epoch:203/300, steps:8/8] -loss: 1.5074 - acc: 39.06%      \n",
      "\n",
      "[epoch:204/300, steps:8/8] -loss: 1.4836 - acc: 42.19%      \n",
      "\n",
      "[epoch:205/300, steps:8/8] -loss: 1.4420 - acc: 45.31%      \n",
      "\n",
      "[epoch:206/300, steps:8/8] -loss: 1.4503 - acc: 45.31%      \n",
      "\n",
      "[epoch:207/300, steps:8/8] -loss: 1.4962 - acc: 40.62%      \n",
      "\n",
      "[epoch:208/300, steps:8/8] -loss: 1.4455 - acc: 45.31%      \n",
      "\n",
      "[epoch:209/300, steps:8/8] -loss: 1.3853 - acc: 51.56%      \n",
      "\n",
      "[epoch:210/300, steps:8/8] -loss: 1.4698 - acc: 42.19%      \n",
      "\n",
      "[epoch:211/300, steps:8/8] -loss: 1.4115 - acc: 48.44%      \n",
      "\n",
      "[epoch:212/300, steps:8/8] -loss: 1.4147 - acc: 48.44%      \n",
      "\n",
      "[epoch:213/300, steps:8/8] -loss: 1.4094 - acc: 48.44%      \n",
      "\n",
      "[epoch:214/300, steps:8/8] -loss: 1.4133 - acc: 46.88%      \n",
      "\n",
      "[epoch:215/300, steps:8/8] -loss: 1.3815 - acc: 51.56%      \n",
      "\n",
      "[epoch:216/300, steps:8/8] -loss: 1.4448 - acc: 45.31%      \n",
      "\n",
      "[epoch:217/300, steps:8/8] -loss: 1.4547 - acc: 43.75%      \n",
      "\n",
      "[epoch:218/300, steps:8/8] -loss: 1.4376 - acc: 46.88%      \n",
      "\n",
      "[epoch:219/300, steps:8/8] -loss: 1.4775 - acc: 42.19%      \n",
      "\n",
      "[epoch:220/300, steps:8/8] -loss: 1.4850 - acc: 42.19%      \n",
      "\n",
      "[epoch:221/300, steps:8/8] -loss: 1.3907 - acc: 50.00%      \n",
      "\n",
      "[epoch:222/300, steps:8/8] -loss: 1.4500 - acc: 45.31%      \n",
      "\n",
      "[epoch:223/300, steps:8/8] -loss: 1.4804 - acc: 42.19%      \n",
      "\n",
      "[epoch:224/300, steps:8/8] -loss: 1.4836 - acc: 42.19%      \n",
      "\n",
      "[epoch:225/300, steps:8/8] -loss: 1.4774 - acc: 42.19%      \n",
      "\n",
      "[epoch:226/300, steps:8/8] -loss: 1.4716 - acc: 45.31%      \n",
      "\n",
      "[epoch:227/300, steps:8/8] -loss: 1.4915 - acc: 40.62%      \n",
      "\n",
      "[epoch:228/300, steps:8/8] -loss: 1.4386 - acc: 48.44%      \n",
      "\n",
      "[epoch:229/300, steps:8/8] -loss: 1.4324 - acc: 46.88%      \n",
      "\n",
      "[epoch:230/300, steps:8/8] -loss: 1.4845 - acc: 40.62%      \n",
      "\n",
      "[epoch:231/300, steps:8/8] -loss: 1.4395 - acc: 45.31%      \n",
      "\n",
      "[epoch:232/300, steps:8/8] -loss: 1.4381 - acc: 45.31%      \n",
      "\n",
      "[epoch:233/300, steps:8/8] -loss: 1.4217 - acc: 48.44%      \n",
      "\n",
      "[epoch:234/300, steps:8/8] -loss: 1.3998 - acc: 50.00%      \n",
      "\n",
      "[epoch:235/300, steps:8/8] -loss: 1.4135 - acc: 48.44%      \n",
      "\n",
      "[epoch:236/300, steps:8/8] -loss: 1.4468 - acc: 45.31%      \n",
      "\n",
      "[epoch:237/300, steps:8/8] -loss: 1.4545 - acc: 45.31%      \n",
      "\n",
      "[epoch:238/300, steps:8/8] -loss: 1.4332 - acc: 45.31%      \n",
      "\n",
      "[epoch:239/300, steps:8/8] -loss: 1.4348 - acc: 46.88%      \n",
      "\n",
      "[epoch:240/300, steps:8/8] -loss: 1.4503 - acc: 45.31%      \n",
      "\n",
      "[epoch:241/300, steps:8/8] -loss: 1.4428 - acc: 45.31%      \n",
      "\n",
      "[epoch:242/300, steps:8/8] -loss: 1.4640 - acc: 43.75%      \n",
      "\n",
      "[epoch:243/300, steps:8/8] -loss: 1.4414 - acc: 46.88%      \n",
      "\n",
      "[epoch:244/300, steps:8/8] -loss: 1.4456 - acc: 45.31%      \n",
      "\n",
      "[epoch:245/300, steps:8/8] -loss: 1.4400 - acc: 46.88%      \n",
      "\n",
      "[epoch:246/300, steps:8/8] -loss: 1.3885 - acc: 51.56%      \n",
      "\n",
      "[epoch:247/300, steps:8/8] -loss: 1.4358 - acc: 46.88%      \n",
      "\n",
      "[epoch:248/300, steps:8/8] -loss: 1.4064 - acc: 50.00%      \n",
      "\n",
      "[epoch:249/300, steps:8/8] -loss: 1.4086 - acc: 50.00%      \n",
      "\n",
      "[epoch:250/300, steps:8/8] -loss: 1.3599 - acc: 53.12%      \n",
      "\n",
      "[epoch:251/300, steps:8/8] -loss: 1.3911 - acc: 51.56%      \n",
      "\n",
      "[epoch:252/300, steps:8/8] -loss: 1.4320 - acc: 46.88%      \n",
      "\n",
      "[epoch:253/300, steps:8/8] -loss: 1.4192 - acc: 48.44%      \n",
      "\n",
      "[epoch:254/300, steps:8/8] -loss: 1.4586 - acc: 43.75%      \n",
      "\n",
      "[epoch:255/300, steps:8/8] -loss: 1.3927 - acc: 50.00%      \n",
      "\n",
      "[epoch:256/300, steps:8/8] -loss: 1.4202 - acc: 48.44%      \n",
      "\n",
      "[epoch:257/300, steps:8/8] -loss: 1.4216 - acc: 48.44%      \n",
      "\n",
      "[epoch:258/300, steps:8/8] -loss: 1.4049 - acc: 51.56%      \n",
      "\n",
      "[epoch:259/300, steps:8/8] -loss: 1.3838 - acc: 51.56%      \n",
      "\n",
      "[epoch:260/300, steps:8/8] -loss: 1.4081 - acc: 50.00%      \n",
      "\n",
      "[epoch:261/300, steps:8/8] -loss: 1.3877 - acc: 51.56%      \n",
      "\n",
      "[epoch:262/300, steps:8/8] -loss: 1.4548 - acc: 43.75%      \n",
      "\n",
      "[epoch:263/300, steps:8/8] -loss: 1.4147 - acc: 48.44%      \n",
      "\n",
      "[epoch:264/300, steps:8/8] -loss: 1.3577 - acc: 54.69%      \n",
      "\n",
      "[epoch:265/300, steps:8/8] -loss: 1.4422 - acc: 45.31%      \n",
      "\n",
      "[epoch:266/300, steps:8/8] -loss: 1.4740 - acc: 43.75%      \n",
      "\n",
      "[epoch:267/300, steps:8/8] -loss: 1.4170 - acc: 48.44%      \n",
      "\n",
      "[epoch:268/300, steps:8/8] -loss: 1.4272 - acc: 48.44%      \n",
      "\n",
      "[epoch:269/300, steps:8/8] -loss: 1.4726 - acc: 43.75%      \n",
      "\n",
      "[epoch:270/300, steps:8/8] -loss: 1.4583 - acc: 43.75%      \n",
      "\n",
      "[epoch:271/300, steps:8/8] -loss: 1.4221 - acc: 48.44%      \n",
      "\n",
      "[epoch:272/300, steps:8/8] -loss: 1.4513 - acc: 43.75%      \n",
      "\n",
      "[epoch:273/300, steps:8/8] -loss: 1.4685 - acc: 43.75%      \n",
      "\n",
      "[epoch:274/300, steps:8/8] -loss: 1.4715 - acc: 42.19%      \n",
      "\n",
      "[epoch:275/300, steps:8/8] -loss: 1.4185 - acc: 48.44%      \n",
      "\n",
      "[epoch:276/300, steps:8/8] -loss: 1.4652 - acc: 43.75%      \n",
      "\n",
      "[epoch:277/300, steps:8/8] -loss: 1.4631 - acc: 43.75%      \n",
      "\n",
      "[epoch:278/300, steps:8/8] -loss: 1.4538 - acc: 45.31%      \n",
      "\n",
      "[epoch:279/300, steps:8/8] -loss: 1.4411 - acc: 45.31%      \n",
      "\n",
      "[epoch:280/300, steps:8/8] -loss: 1.4849 - acc: 42.19%      \n",
      "\n",
      "[epoch:281/300, steps:8/8] -loss: 1.4519 - acc: 45.31%      \n",
      "\n",
      "[epoch:282/300, steps:8/8] -loss: 1.4645 - acc: 43.75%      \n",
      "\n",
      "[epoch:283/300, steps:8/8] -loss: 1.4404 - acc: 45.31%      \n",
      "\n",
      "[epoch:284/300, steps:8/8] -loss: 1.4341 - acc: 46.88%      \n",
      "\n",
      "[epoch:285/300, steps:8/8] -loss: 1.4580 - acc: 45.31%      \n",
      "\n",
      "[epoch:286/300, steps:8/8] -loss: 1.4380 - acc: 45.31%      \n",
      "\n",
      "[epoch:287/300, steps:8/8] -loss: 1.4590 - acc: 43.75%      \n",
      "\n",
      "[epoch:288/300, steps:8/8] -loss: 1.4148 - acc: 48.44%      \n",
      "\n",
      "[epoch:289/300, steps:8/8] -loss: 1.4146 - acc: 50.00%      \n",
      "\n",
      "[epoch:290/300, steps:8/8] -loss: 1.4748 - acc: 43.75%      \n",
      "\n",
      "[epoch:291/300, steps:8/8] -loss: 1.4915 - acc: 40.62%      \n",
      "\n",
      "[epoch:292/300, steps:8/8] -loss: 1.4634 - acc: 43.75%      \n",
      "\n",
      "[epoch:293/300, steps:8/8] -loss: 1.5265 - acc: 37.50%      \n",
      "\n",
      "[epoch:294/300, steps:8/8] -loss: 1.5244 - acc: 37.50%      \n",
      "\n",
      "[epoch:295/300, steps:8/8] -loss: 1.4647 - acc: 43.75%      \n",
      "\n",
      "[epoch:296/300, steps:8/8] -loss: 1.5361 - acc: 35.94%      \n",
      "\n",
      "[epoch:297/300, steps:8/8] -loss: 1.5240 - acc: 37.50%      \n",
      "\n",
      "[epoch:298/300, steps:8/8] -loss: 1.4730 - acc: 43.75%      \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[epoch:299/300, steps:8/8] -loss: 1.4644 - acc: 43.75%      \n",
      "\n",
      "[epoch:300/300, steps:8/8] -loss: 1.4646 - acc: 42.19%      \n",
      "\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 300\n",
    "batch_size = 8\n",
    "n_batch = int(X_train.shape[0] / batch_size)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "lr=0.0005\n",
    "print('train for f_weights...')\n",
    "for epoch in range(n_epochs):\n",
    "#     if epoch%100==0 and epoch!=0:\n",
    "#         lr*=0.5\n",
    "    # training step\n",
    "    index = np.arange(X_train.shape[0])\n",
    "    np.random.shuffle(index)\n",
    "    X_train = X_train[index]\n",
    "    y_train = y_train[index]\n",
    "    sum_loss = 0\n",
    "    last_train_str = \"\"\n",
    "    for i in range(n_batch):\n",
    "        x_path = X_train[i * batch_size:(i + 1) * batch_size]\n",
    "        x_temp = []\n",
    "        for path in x_path:\n",
    "            x_i= pd.read_csv(path)\n",
    "            x_i.pop('id')\n",
    "            x_temp.append(x_i.values)\n",
    "        x_temp = np.reshape(x_temp, (batch_size, timesteps, dim))\n",
    "        y_temp = y_train[i * batch_size:(i + 1) * batch_size]\n",
    "        y_temp = np.reshape(y_temp, (batch_size, 1))\n",
    "        feed_dict = {inputs: x_temp, y: y_temp}\n",
    "        _, loss_value = sess.run([train_op1, loss1], feed_dict=feed_dict)\n",
    "        sum_loss += loss_value\n",
    "        last_train_str = \"\\r[epoch:%d/%d, steps:%d/%d] -loss: %.4f\" % \\\n",
    "                         (epoch + 1, n_epochs, i + 1, n_batch, sum_loss / (i + 1))\n",
    "        print(last_train_str, end='      ', flush=True)\n",
    "    print('\\n')\n",
    "\n",
    "print('train for the attention model...')\n",
    "lr=0.0005\n",
    "for epoch in range(n_epochs):\n",
    "    if epoch%10==0 and epoch!=0:\n",
    "        lr*=0.5\n",
    "    index = np.arange(X_train.shape[0])\n",
    "    np.random.shuffle(index)\n",
    "    X_train = X_train[index]\n",
    "    y_train = y_train[index]\n",
    "    sum_loss = 0\n",
    "    sum_acc = 0\n",
    "    last_train_str = \"\"\n",
    "    for i in range(n_batch):\n",
    "        x_path = X_train[i * batch_size:(i + 1) * batch_size]\n",
    "        x_temp = []\n",
    "        for path in x_path:\n",
    "            x_i= pd.read_csv(path)\n",
    "            x_i.pop('id')\n",
    "            x_temp.append(x_i.values)\n",
    "        x_temp = np.reshape(x_temp, (batch_size, timesteps, dim))\n",
    "        y_temp = y_train[i * batch_size:(i + 1) * batch_size]\n",
    "        y_temp = np.reshape(y_temp, (batch_size, 1))\n",
    "        feed_dict = {inputs: x_temp, y: y_temp}\n",
    "        _, loss_value, acc_value = sess.run([train_op2, loss2, accuracy], feed_dict=feed_dict)\n",
    "        sum_loss += loss_value\n",
    "        sum_acc += (acc_value*100)\n",
    "        last_train_str = \"\\r[epoch:%d/%d, steps:%d/%d] -loss: %.4f - acc: %.2f%%\" % \\\n",
    "                         (epoch + 1, n_epochs, i + 1, n_batch, sum_loss / (i + 1), sum_acc / (i + 1))\n",
    "        print(last_train_str, end='      ', flush=True)\n",
    "    print('\\n')\n",
    "    \n",
    "# print('train for the simple lstm model...')\n",
    "# lr=0.01\n",
    "# for epoch in range(n_epochs):\n",
    "# #     if epoch%10==0 and epoch!=0:\n",
    "# #         lr*=0.5\n",
    "#     index = np.arange(X_train.shape[0])\n",
    "#     np.random.shuffle(index)\n",
    "#     X_train = X_train[index]\n",
    "#     y_train = y_train[index]\n",
    "#     sum_loss = 0\n",
    "#     sum_acc = 0\n",
    "#     last_train_str = \"\"\n",
    "#     for i in range(n_batch):\n",
    "#         x_path = X_train[i * batch_size:(i + 1) * batch_size]\n",
    "#         x_temp = []\n",
    "#         for path in x_path:\n",
    "#             x_i= pd.read_csv(path)\n",
    "#             x_i.pop('id')\n",
    "#             x_temp.append(x_i.values)\n",
    "#         x_temp = np.reshape(x_temp, (batch_size, timesteps, dim))\n",
    "#         y_temp = y_train[i * batch_size:(i + 1) * batch_size]\n",
    "#         y_temp = np.reshape(y_temp, (batch_size, 1))\n",
    "#         feed_dict = {inputs: x_temp, y: y_temp}\n",
    "#         _, loss_value, acc_value = sess.run([train_op3, loss3, accuracy], feed_dict=feed_dict)\n",
    "#         sum_loss += loss_value\n",
    "#         sum_acc += (acc_value*100)\n",
    "#         last_train_str = \"\\r[epoch:%d/%d, steps:%d/%d] -loss: %.4f - acc: %.2f%%\" % \\\n",
    "#                          (epoch + 1, n_epochs, i + 1, n_batch, sum_loss / (i + 1), sum_acc / (i + 1))\n",
    "#         print(last_train_str, end='      ', flush=True)\n",
    "#     print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
